from data_structs import Tweet
from data_structs import Text
import urllib

def crawl_url(one_tweet):
    
    url = one_tweet.get_url()
    try:
        page = urllib.urlopen(url)
        page_text = page.read().decode('UTF-8')
    except Exception as e:
        text = Text("URL", "")   
        print(url + ':' + str(e))
        return text
    
    start_index = page_text.find("venue: {")
    if start_index == -1:
        text = Text("URL", "")
        return text
    else:
        start_index = start_index + 7
        end_index = start_index + 1
        while page_text[end_index] != '\n':
            end_index = end_index + 1
        end_index = end_index - 1
        json_string = page_text[start_index:end_index]
        text = Text("URL", json_string)    
        return text
    